{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "\n",
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor import problems\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "tf.logging.set_verbosity(tf.logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "vocab = 'sp10m.cased.t5.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)\n",
    "\n",
    "class Encoder:\n",
    "    def __init__(self, sp):\n",
    "        self.sp = sp\n",
    "        self.vocab_size = sp.GetPieceSize() + 100\n",
    "    \n",
    "    def encode(self, s):\n",
    "        return self.sp.EncodeAsIds(s)\n",
    "    \n",
    "    def decode(self, ids, strip_extraneous=False):\n",
    "        return self.sp.DecodeIds(list(ids))\n",
    "    \n",
    "encoder = Encoder(sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "\n",
    "@registry.register_problem\n",
    "class Seq2Seq(text_problems.Text2TextProblem):\n",
    "\n",
    "    @property\n",
    "    def approx_vocab_size(self):\n",
    "        return 32100\n",
    "    \n",
    "    @property\n",
    "    def is_generate_per_split(self):\n",
    "        return False\n",
    "            \n",
    "    def feature_encoders(self, data_dir):\n",
    "        encoder = Encoder(sp)\n",
    "        return {\n",
    "            \"inputs\": encoder,\n",
    "            \"targets\": encoder\n",
    "        }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = os.path.expanduser('t2t2/data')\n",
    "TMP_DIR = os.path.expanduser('t2t2/tmp')\n",
    "TRAIN_DIR = os.path.expanduser('t2t2/train-base')\n",
    "EXPORT_DIR = os.path.expanduser('t2t2/export')\n",
    "TRANSLATIONS_DIR = os.path.expanduser('t2t2/translation')\n",
    "EVENT_DIR = os.path.expanduser('t2t2/event')\n",
    "USR_DIR = os.path.expanduser('t2t2/user')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROBLEM = 'seq2_seq'\n",
    "t2t_problem = problems.problem(PROBLEM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'base/model.ckpt-300000'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "ckpt_path = 'base/model.ckpt-300000'\n",
    "ckpt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tensor2tensor import models\n",
    "from tensor2tensor import problems\n",
    "from tensor2tensor.layers import common_layers\n",
    "from tensor2tensor.utils import trainer_lib\n",
    "from tensor2tensor.utils import t2t_model\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor.utils import metrics\n",
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'train'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'train'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7f470d042268> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7f470d042268> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function framework at 0x7f470d042268> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:2253: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:2253: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:1374: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:1374: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7f470c6a5488> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7f470c6a5488> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function attention_bias_to_padding at 0x7f470c6a5488> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7f470d000f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7f46d01542b0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7f470d000f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7f46d01542b0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function layers at 0x7f470d000f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7f46d01542b0>\n",
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:1258: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:1258: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from base/model.ckpt-300000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from base/model.ckpt-300000\n"
     ]
    }
   ],
   "source": [
    "class Model:\n",
    "    def __init__(self, HPARAMS = \"transformer_base\", DATA_DIR = 't2t/data'):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        \n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        maxlen_decode = tf.reduce_max(self.X_seq_len)\n",
    "        \n",
    "        x = tf.expand_dims(tf.expand_dims(self.X, -1), -1)\n",
    "        y = tf.expand_dims(tf.expand_dims(self.Y, -1), -1)\n",
    "        \n",
    "        features = {\n",
    "            \"inputs\": x,\n",
    "            \"targets\": y,\n",
    "            \"target_space_id\": tf.constant(1, dtype=tf.int32),\n",
    "        }\n",
    "        self.features = features\n",
    "        \n",
    "        Modes = tf.estimator.ModeKeys\n",
    "        hparams = trainer_lib.create_hparams(HPARAMS, data_dir=DATA_DIR, problem_name=PROBLEM)\n",
    "        hparams.filter_size = 3072\n",
    "        hparams.hidden_size = 768\n",
    "        hparams.num_heads = 12\n",
    "        hparams.num_hidden_layers = 8\n",
    "        hparams.vocab_divisor = 128\n",
    "        hparams.label_smoothing = 0.0\n",
    "        hparams.shared_embedding_and_softmax_weights = False\n",
    "        hparams.dropout = 0.1\n",
    "        hparams.max_length = 1024\n",
    "        hparams.multiproblem_mixing_schedule = \"pretrain\"\n",
    "\n",
    "        hparams.optimizer = \"Adafactor\"\n",
    "        hparams.learning_rate_warmup_steps = 10000\n",
    "        hparams.learning_rate_schedule = \"rsqrt_decay\"\n",
    "        \n",
    "        translate_model = registry.model('transformer')(hparams, Modes.TRAIN)\n",
    "        self.translate_model = translate_model\n",
    "        logits, _ = translate_model(features)\n",
    "        self.logits = logits\n",
    "        \n",
    "        with tf.variable_scope(tf.get_variable_scope(), reuse=True):\n",
    "            self.fast_result = translate_model._greedy_infer(features, maxlen_decode)[\"outputs\"]\n",
    "            self.beam_result = translate_model._beam_decode_slow(\n",
    "                features, maxlen_decode, beam_size=5, \n",
    "                top_beams=1, alpha=1.0)[\"outputs\"]\n",
    "        \n",
    "        self.fast_result = tf.identity(self.fast_result, name = 'greedy')\n",
    "        self.beam_result = tf.identity(self.beam_result, name = 'beam')\n",
    "        \n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, ckpt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'b2b-base/model.ckpt'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# saver = tf.train.Saver(tf.trainable_variables())\n",
    "# saver.save(sess, 'b2b-base/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rm -rf b2b-base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dumping-watpadd.txt.tsv',\n",
       " 'dumping-wiki.txt.tsv',\n",
       " 'dumping-instagram.txt.tsv',\n",
       " 'dumping-parliament.txt.tsv',\n",
       " 'dumping-pdf.txt.tsv',\n",
       " 'dumping-news.txt.tsv',\n",
       " 'dumping-iium.txt.tsv']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glob('dumping*txt.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/t5/models/mesh_transformer.py:210: UserWarning: get_sentencepiece_model_path is deprecated. Please pass the mixture or task vocabulary directly to the Mesh TensorFlow Transformer instead.\n",
      "  \"get_sentencepiece_model_path is deprecated. Please pass the mixture or \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:tokens_length=1137 inputs_length=1024 targets_length=229 noise_density=0.15 mean_noise_span_length=3.0 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:tokens_length=1137 inputs_length=1024 targets_length=229 noise_density=0.15 mean_noise_span_length=3.0 \n"
     ]
    }
   ],
   "source": [
    "from t5.data import preprocessors as prep\n",
    "import functools\n",
    "import t5\n",
    "import gin\n",
    "\n",
    "gin.parse_config_file('pretrained_models_base_operative_config.gin')\n",
    "\n",
    "def dumping_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        ['dumping-wiki.txt.tsv']\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['title', 'text'], ex)))\n",
    "    return ds\n",
    "\n",
    "t5.data.TaskRegistry.remove('dumping_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'dumping_dataset',\n",
    "    dataset_fn = dumping_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = functools.partial(\n",
    "        t5.data.preprocessors.rekey,\n",
    "        key_map = {'inputs': None, 'targets': 'text'},\n",
    "    ),\n",
    "    token_preprocessor = t5.data.preprocessors.unsupervised,\n",
    "    sentencepiece_model_path = vocab,\n",
    "    metric_fns = [],\n",
    ")\n",
    "\n",
    "nq_task = t5.data.TaskRegistry.get(\"dumping_dataset\")\n",
    "ds = nq_task.get_dataset(split='qa.tsv', sequence_length={\"inputs\": 768, \"targets\": 768})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function neighboring_pairs.<locals>.<lambda> at 0x7f43d5d3d9d8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Failed to parse source code of <function neighboring_pairs.<locals>.<lambda> at 0x7f43d5d3d9d8>, which Python reported as:\n",
      "      lambda x: x[text_key], num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
      "\n",
      "If this is a lambda function, the error may be avoided by creating the lambda in a standalone statement.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function neighboring_pairs.<locals>.<lambda> at 0x7f43d5d3d9d8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Failed to parse source code of <function neighboring_pairs.<locals>.<lambda> at 0x7f43d5d3d9d8>, which Python reported as:\n",
      "      lambda x: x[text_key], num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
      "\n",
      "If this is a lambda function, the error may be avoided by creating the lambda in a standalone statement.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function neighboring_pairs.<locals>.<lambda> at 0x7f43d5d3d9d8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Failed to parse source code of <function neighboring_pairs.<locals>.<lambda> at 0x7f43d5d3d9d8>, which Python reported as:\n",
      "      lambda x: x[text_key], num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
      "\n",
      "If this is a lambda function, the error may be avoided by creating the lambda in a standalone statement.\n"
     ]
    }
   ],
   "source": [
    "def pair_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        glob('dumping*pair.tsv')\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['text'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('pair_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'pair_dataset',\n",
    "    dataset_fn = pair_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [prep.next_sentence_prediction],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")\n",
    "\n",
    "nq_task = t5.data.TaskRegistry.get(\"pair_dataset\")\n",
    "ds = nq_task.get_dataset(split='qa.tsv', sequence_length={\"inputs\": 768, \"targets\": 768})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "import numpy as np\n",
    "\n",
    "iterator = tfds.as_numpy(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tentera daripada kekuasaannya dan menyebabkan Khalaf terpaksa mengalahkan.amir.M pertempuran sehingga kekuasaan Khalf tegak kembali berakhirf. www.iranchamber.com padarieilung Verkehrswesen 7 Perang Penggangkut) sebagai Jerman semasa Perang Dunia pengendali A708/15 7 5. hadapan. Ia tidak jelas berapa banyak hanya kerangka mm di bukannya kepingan perisai Ini menawarkan perlindungan perisai kereta kebal menghasilkan 100 kuasa kuda ( km (132 gelen. Ia seperlahan kereta dari bererti kawasan. Bagaimanapun di( keupayaan diu 24 April 1918 apabila sengaja kebal dan satu kereta kebal Jantan dengan meriam enam kebal hadapan, 2nd Lt mesingan berundur rosak terkena kereta kebalnya dan memusnahkannya. Dia sebanyak tiga kali, dan membunuh lima krew untuk Dua A7 L L pasukan dua terbalik kedalam lubang, sesetengah mengalami ke dapat digunakan dan dilupuskan A'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = next(iterator)\n",
    "encoder.decode([i for i in r['targets'].tolist() if i < 32000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = sess.run(model.logits, feed_dict = {model.X: [r['inputs']], model.Y: [r['targets']]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dari kerajaan keatas denganaf terpaksa membalas.\"mir. 9 Pertempuran Khal Abu Khalf berakhirf Dinasti(.tsjmads Termasuk pada tahunlausseklehr\"\" Keubahsuaigal) sebagai Perang Perang Dunia belas dan A7,,h 8. hadapan yang A banyak bahawa gun mm, tidak perisai perisai, Senjata adalah kepingan  kuasa kW tembakan (3 km (atau tan m pada Tiada boleht keretanya, parit. Ia kereta( dan dengan hadapan pertembungan Mac 1917, sengaja kebal infantri kebal kebal, kebal A mendahului sungguhpunring. kebal... kebal palang dengan lebih lebih menolak lebih lebih.t. kebal kereta. medan kelihatannya. lebih'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.decode([i for i in np.argmax(f[0,:,0,0], axis = -1).tolist() if i < 32000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([32099,    37, 32098, 32098,    72, 32097, 14018,    22, 32097,\n",
       "          13,   492,  1060,  6390, 32097,     3, 32096,     6,  6160,\n",
       "           3, 32095,   482, 32094,  4141, 12529,  1048, 12529, 32093,\n",
       "         492, 32092, 32092, 32092,  1902, 32091,   492, 32090,  4869,\n",
       "           4,     3,  2268,   963,  4407,    25, 16216, 32090,    23,\n",
       "          31,  4938, 32088,  8822, 32087, 32087, 15562, 32087, 32087,\n",
       "        5253, 32087,     6,     6, 32086,   309, 26823,  4832, 32085,\n",
       "           5, 32084,    41, 32083,  1000, 32082,  1000,   653, 32082,\n",
       "        2961,    16,   128,   376, 32080,    14,    14,   211, 32079,\n",
       "       32079,   410,     3, 32078,   942, 32077,    17,   128, 32076,\n",
       "       32076, 32076,   127, 32076,    90, 32075, 13654,  4791,    14,\n",
       "       32074,    29,  6925,  6925,    14, 10922,    35, 32072, 32072,\n",
       "       13085, 32071, 32071, 32071,    15,   284, 22331,  4733,    15,\n",
       "           4,   205,  1468, 32069,    15,     4,  3096,  2298,   856,\n",
       "       32068,    23,  2993,    66,    81,   542, 32067,    26, 32066,\n",
       "          14, 32065, 11335, 32064,     3,   132, 32063,   542, 32062,\n",
       "           4, 32061,    16, 32060,    22,   942, 32059,  8445,   471,\n",
       "       12285,    14, 32058,  3227, 32057,  2506, 12585,  2506, 32055,\n",
       "       32055,  2506, 32055,    14, 32055, 32055, 32055,  2506,   128,\n",
       "        7389,  9081, 32054,  2850,     3, 32054,  2506, 32053, 32053,\n",
       "           3,     3, 32052,     3,  2506, 32051, 32051, 32051, 24397,\n",
       "          22, 32051, 32051,    51, 32050, 32050, 32050, 32050, 32050,\n",
       "       32050, 32050, 32050,    51,  1390,    51, 32048, 32048, 32048,\n",
       "          51, 32047,     3,    81,     3, 32045,  2506,   542,     3,\n",
       "        3714, 32044, 32044, 20795, 32044, 32044,     3, 32043, 32043,\n",
       "       32043, 32043,    51,     1])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(f[0,:,0,0], axis = -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([32099,   460,    47, 32098,  4806,    26,    16,   370, 12529,\n",
       "          13,   492,  1060,  4087, 32097,     3, 32096,    13,  6160,\n",
       "           3, 32095,   238, 32094,  2815,   135,  4806, 12529, 32093,\n",
       "         492, 16580,   448, 32092,  1902, 32091,   492,     3,    15,\n",
       "       28759,     3,  5691, 21240,   839,     3,  1065, 32090,    23,\n",
       "       32089,  6691, 32088,   752,  1086,  3376, 18098,  3923,  3814,\n",
       "        1840, 32087,   362,  1000, 32086,  1639,  1255, 17040, 32085,\n",
       "           5, 32084,    41, 32083,   426,   251,  1000,   653, 32082,\n",
       "        4331, 32081,   128,   376, 32080,  3052,   214,   871,   362,\n",
       "       32079,   241,     3, 32078,   942, 32077,     3,   132,    29,\n",
       "         740,  3620,   127, 32076,   121,  5553, 32075,  4791,    18,\n",
       "       32074,  2282, 13085,  6925, 32073,   297,  1342,  1745, 32072,\n",
       "        6925,   542,  2506, 32071,   803,  1026,   284,  3971,    15,\n",
       "           4, 32070,  1468, 32069,    15,     4,  1009,   143, 20578,\n",
       "       32068,     3,   132, 17743,  6674,   542, 32067,    37, 32066,\n",
       "        5444, 32065,    61, 32064,     3,  1010, 32063,    18, 32062,\n",
       "           4, 32061,  2091, 32060,    18,   198, 32059,   845,   452,\n",
       "       11318,   169, 32058,  3227, 32057,  2506, 32056,    16,    55,\n",
       "         542,  2506, 26351,    22,  4553,   810, 32055,  2506,   942,\n",
       "          14,   126,  3679,   735,    81, 32054, 14439,  6796, 32053,\n",
       "        4500,  5857, 32052,   542,  2506,    26,    16,  4897,    26,\n",
       "           3,   706, 32051,   147,   212,   252,    14,    16,  1888,\n",
       "         487, 14267, 32050,    21, 32049,  1698,   128,   376, 32048,\n",
       "         735, 32047,   735, 32046,   276, 32045,   102, 11222, 15077,\n",
       "        7378,    14,  1703,   851,    30, 32044,    91,   130,    16,\n",
       "       18147, 32043,   128,     1])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r['targets']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = sess.run(n)\n",
    "r[0]['inputs'][:,:,0,0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r[0]['inputs'][:,:,0,0][-3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r[0]['targets'][:10,:,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams.data_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r['targets'], np.argmax(f[0,:,0,0], axis = -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder.decode(np.argmax(f[0,:,0,0], axis = -1).tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    return re.sub(r'[ ]+', ' ', unidecode(string)).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = \"\"\"\n",
    "SHAH ALAM - Langkah Tun Dr Mahathir Mohamad yang mahu menubuhkan parti baharu sekali lagi dilihat akan memecahbelahkan orang Melayu, kata bekas pegawai khas Datuk Seri Najib Tun Razak, Isham Jalil.\n",
    "\n",
    "Dakwa beliau, orang Melayu akan menjadi semakin lemah akibat tindakan bekas Perdana Menteri itu.\n",
    "\n",
    "\"Lebih dua puluh tahun dahulu, Dr Mahathir telah memecahbelahkan orang-orang Melayu dengan penubuhan Parti Keadilan Rakyat (PKR) oleh Datuk Seri Anwar Ibrahim akibat pergaduhan Dr Mahathir dengan beliau.\n",
    "\n",
    "\"Lima tahun lalu juga, Dr Mahathir telah memecahbelahkan orang-orang Melayu melalui penubuhan Bersatu untuk menjatuhkan Najib kerana tidak membantu Mukhriz dalam pemilihan UMNO serta tidak membantu kroni-kroni beliau untuk menyambung kontrak dan konsesi kerajaan yang diperoleh selama berpuluh tahun sebelum itu.\n",
    "\n",
    "\"Ini tidak akan berhenti sehinggalah Mukhriz menjadi Perdana Menteri dan kepentingan peribadi serta kroni-kroninya dipenuhi,\" katanya dalam satu kenyataan hari ini.\n",
    "\n",
    "Justeru katanya, Ahli Parlimen Langkawi itu layak dilabelkan sebagai 'Bapa Pemecahbelah Orang-orang Melayu'.\n",
    "\n",
    "\"Parti baharu Dr Mahathir ini dikhabarkan yang terdiri daripada serpihan ahli-ahli Bersatu dalam kalangan orang-orang Melayu.\n",
    "\n",
    "\"Sebab jika ada banyak parti Melayu, orang-orang Melayu berkemungkinan besar akan lebih bertelagah dan berpecah belah,\" katanya.\n",
    "\n",
    "Isham yang juga Ketua Penerangan Barisan Nasional (BN) Selangor berkata, musuh Melayu dan DAP kini paling beruntung dan bersorak dengan tindakan Dr Mahathir ini.\n",
    "\n",
    "\"Jika masih ada lagi orang-orang Melayu yang masih tidak sedar dan masih mengikut Dr Mahathir, saya tidak tahu nak kata apa.\n",
    "\n",
    "\"Kepada Dr Mahathir dan pengikut-pengikutnya ini, daripada mereka menjadi musuh dalam selimut untuk orang-orang Melayu, mungkin lebih baik mereka terus masuk DAP secara terang-terangan seperti segelintir Melayu DAP yang lain.\n",
    "\n",
    "\"Dengan ini mereka tidak perlu menyorok atau berselindung lagi tentang perjuangan sebenar mereka,\" katanya.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(f'tajuk: {cleaning(string)}') + [1]\n",
    "f = sess.run(model.fast_result, feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(f'ringkasan: {cleaning(string)}') + [1]\n",
    "f = sess.run(model.fast_result, feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(f'soalan: siapakah perdana menteri malaysia?') + [1]\n",
    "f = sess.run(model.fast_result, feed_dict = {model.X: [encoded]})\n",
    "encoder.decode(f[0].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
